[FRAUD] 데이터정리 시도(GCN_엣지손x)

Author

김보람

Published

October 20, 2023

imports

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt 
import networkx as nx
import sklearn
import xgboost as xgb

# sklearn
from sklearn import model_selection # split함수이용
from sklearn import ensemble # RF,GBM
from sklearn import metrics
from sklearn.metrics import precision_score, recall_score, f1_score
from sklearn.svm import SVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import GaussianNB

# gnn
import torch
import torch.nn.functional as F
import torch_geometric
from torch_geometric.nn import GCNConv

/home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/torch_geometric/typing.py:18: UserWarning: An issue occurred while importing 'pyg-lib'. Disabling its usage. Stacktrace: /home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/libpyg.so: undefined symbol: _ZN2at4_ops12split_Tensor4callERKNS_6TensorEN3c106SymIntEl
  warnings.warn(f"An issue occurred while importing 'pyg-lib'. "
/home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/torch_geometric/typing.py:31: UserWarning: An issue occurred while importing 'torch-scatter'. Disabling its usage. Stacktrace: /home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/torch_scatter/_scatter_cuda.so: undefined symbol: _ZNK3c107SymBool10guard_boolEPKcl
  warnings.warn(f"An issue occurred while importing 'torch-scatter'. "
/home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/torch_geometric/typing.py:42: UserWarning: An issue occurred while importing 'torch-sparse'. Disabling its usage. Stacktrace: /home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/torch_sparse/_diag_cuda.so: undefined symbol: _ZN3c106detail19maybe_wrap_dim_slowIlEET_S2_S2_b
  warnings.warn(f"An issue occurred while importing 'torch-sparse'. "
def down_sample_textbook(df):
    df_majority = df[df.is_fraud==0].copy()
    df_minority = df[df.is_fraud==1].copy()
    df_maj_dowsampled = sklearn.utils.resample(df_majority, n_samples=len(df_minority), replace=False, random_state=42)
    df_downsampled = pd.concat([df_minority, df_maj_dowsampled])
    return df_downsampled

def compute_time_difference(group):
    n = len(group)
    result = []
    for i in range(n):
        for j in range(n):
            time_difference = abs(group.iloc[i].trans_date_trans_time.value - group.iloc[j].trans_date_trans_time.value)
            result.append([group.iloc[i].name, group.iloc[j].name, time_difference])
    return result

def mask(df):
    df_tr,df_test = sklearn.model_selection.train_test_split(df, random_state=42)
    N = len(df)
    train_mask = [i in df_tr.index for i in range(N)]
    test_mask = [i in df_test.index for i in range(N)]
    train_mask = np.array(train_mask)
    test_mask = np.array(test_mask)
    return train_mask, test_mask

def edge_index_selected(edge_index):
    theta = edge_index[:,2].mean()
    edge_index[:,2] = (np.exp(-edge_index[:,2]/theta) != 1)*(np.exp(-edge_index[:,2]/theta))
    edge_index = edge_index.tolist()
    mean_ = np.array(edge_index)[:,2].mean()
    selected_edges = [(int(row[0]), int(row[1])) for row in edge_index if row[2] > mean_]
    edge_index_selected = torch.tensor(selected_edges, dtype=torch.long).t()
    return edge_index_selected


fraudTrain = pd.read_csv("~/Desktop/fraudTrain.csv").iloc[:,1:]
fraudTrain = fraudTrain.assign(trans_date_trans_time= list(map(lambda x: pd.to_datetime(x), fraudTrain.trans_date_trans_time)))
fraudTrain
trans_date_trans_time cc_num merchant category amt first last gender street city ... lat long city_pop job dob trans_num unix_time merch_lat merch_long is_fraud
0 2019-01-01 00:00:00 2.703190e+15 fraud_Rippin, Kub and Mann misc_net 4.97 Jennifer Banks F 561 Perry Cove Moravian Falls ... 36.0788 -81.1781 3495 Psychologist, counselling 1988-03-09 0b242abb623afc578575680df30655b9 1325376018 36.011293 -82.048315 0
1 2019-01-01 00:00:00 6.304230e+11 fraud_Heller, Gutmann and Zieme grocery_pos 107.23 Stephanie Gill F 43039 Riley Greens Suite 393 Orient ... 48.8878 -118.2105 149 Special educational needs teacher 1978-06-21 1f76529f8574734946361c461b024d99 1325376044 49.159047 -118.186462 0
2 2019-01-01 00:00:00 3.885950e+13 fraud_Lind-Buckridge entertainment 220.11 Edward Sanchez M 594 White Dale Suite 530 Malad City ... 42.1808 -112.2620 4154 Nature conservation officer 1962-01-19 a1a22d70485983eac12b5b88dad1cf95 1325376051 43.150704 -112.154481 0
3 2019-01-01 00:01:00 3.534090e+15 fraud_Kutch, Hermiston and Farrell gas_transport 45.00 Jeremy White M 9443 Cynthia Court Apt. 038 Boulder ... 46.2306 -112.1138 1939 Patent attorney 1967-01-12 6b849c168bdad6f867558c3793159a81 1325376076 47.034331 -112.561071 0
4 2019-01-01 00:03:00 3.755340e+14 fraud_Keeling-Crist misc_pos 41.96 Tyler Garcia M 408 Bradley Rest Doe Hill ... 38.4207 -79.4629 99 Dance movement psychotherapist 1986-03-28 a41d7549acf90789359a9aa5346dcb46 1325376186 38.674999 -78.632459 0
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
1048570 2020-03-10 16:07:00 6.011980e+15 fraud_Fadel Inc health_fitness 77.00 Haley Wagner F 05561 Farrell Crescent Annapolis ... 39.0305 -76.5515 92106 Accountant, chartered certified 1943-05-28 45ecd198c65e81e597db22e8d2ef7361 1362931649 38.779464 -76.317042 0
1048571 2020-03-10 16:07:00 4.839040e+15 fraud_Cremin, Hamill and Reichel misc_pos 116.94 Meredith Campbell F 043 Hanson Turnpike Hedrick ... 41.1826 -92.3097 1583 Geochemist 1999-06-28 c00ce51c6ebb7657474a77b9e0b51f34 1362931670 41.400318 -92.726724 0
1048572 2020-03-10 16:08:00 5.718440e+11 fraud_O'Connell, Botsford and Hand home 21.27 Susan Mills F 005 Cody Estates Louisville ... 38.2507 -85.7476 736284 Engineering geologist 1952-04-02 17c9dc8b2a6449ca2473726346e58e6c 1362931711 37.293339 -84.798122 0
1048573 2020-03-10 16:08:00 4.646850e+18 fraud_Thompson-Gleason health_fitness 9.52 Julia Bell F 576 House Crossroad West Sayville ... 40.7320 -73.1000 4056 Film/video editor 1990-06-25 5ca650881b48a6a38754f841c23b77ab 1362931718 39.773077 -72.213209 0
1048574 2020-03-10 16:08:00 2.283740e+15 fraud_Buckridge PLC misc_pos 6.81 Shannon Williams F 9345 Spencer Junctions Suite 183 Alpharetta ... 34.0770 -84.3033 165556 Prison officer 1997-12-27 8d0a575fe635bbde12f1a2bffc126731 1362931730 33.601468 -83.891921 0

1048575 rows × 22 columns

데이터정리

_df1 = fraudTrain[fraudTrain["is_fraud"] == 0].sample(frac=0.20, random_state=42)
_df2 = fraudTrain[fraudTrain["is_fraud"] == 1]
df02 = pd.concat([_df1,_df2])
df02.shape
(214520, 22)
df50 = down_sample_textbook(df02)
df50 = df50.reset_index()
df50.shape
(12012, 23)

tr/test

mask(df50)
(array([False,  True,  True, ...,  True, False,  True]),
 array([ True, False, False, ..., False,  True, False]))
train_mask, test_mask = mask(df50)

edge_index 설정

# groups = df50.groupby('cc_num')
# edge_index_list_plus = [compute_time_difference(group) for _, group in groups]
# edge_index_list_plus_flat = [item for sublist in edge_index_list_plus for item in sublist]
# edge_index_list_plus_nparr = np.array(edge_index_list_plus_flat)
# np.save('edge_index_list_plus50.npy', edge_index_list_plus_nparr)
edge_index = np.load('edge_index_list_plus50.npy').astype(np.float64)
edge_index.shape
(200706, 3)
edge_index
array([[1.023000e+03, 1.023000e+03, 0.000000e+00],
       [1.023000e+03, 1.024000e+03, 4.200000e+12],
       [1.023000e+03, 1.028000e+03, 7.764000e+13],
       ...,
       [1.194400e+04, 9.782000e+03, 9.528000e+13],
       [1.194400e+04, 1.176700e+04, 3.055758e+16],
       [1.194400e+04, 1.194400e+04, 0.000000e+00]])
groups = df50.groupby('cc_num')
groups
<pandas.core.groupby.generic.DataFrameGroupBy object at 0x7f8740677af0>
import itertools

index_pairs = []

for _, group_df in groups:
    indices = group_df.index.tolist()
    index_combinations = list(itertools.combinations(indices, 2))
    index_pairs.extend(index_combinations)
index_pairs = torch.tensor(index_pairs).t()
index_pairs.shape
torch.Size([2, 94347])
index_pairs
tensor([[ 1023,  1023,  1023,  ...,  9782,  9782, 11767],
        [ 1024,  1028,  1031,  ..., 11767, 11944, 11944]])
# edge_index_selected = edge_index_selected(edge_index)

data설정(x, edge_index, y)

x = torch.tensor(df50['amt'], dtype=torch.float).reshape(-1,1)
y = torch.tensor(df50['is_fraud'],dtype=torch.int64)
data = torch_geometric.data.Data(x=x, edge_index = index_pairs, y=y, train_mask = train_mask, test_mask = test_mask)
data

Data(x=[12012, 1], edge_index=[2, 94347], y=[12012], train_mask=[12012], test_mask=[12012])

분석 1(GCN): amt

torch.manual_seed(202250926)
class GCN2(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = GCNConv(1, 32)
        self.conv2 = GCNConv(32,2)

    def forward(self, data):
        x, edge_index = data.x, data.edge_index

        x = self.conv1(x, edge_index)
        x = F.relu(x)
        x = F.dropout(x, training=self.training)
        x = self.conv2(x, edge_index)

        return F.log_softmax(x, dim=1)

X = (data.x[data.train_mask]).numpy()
XX = (data.x[data.test_mask]).numpy()
y = (data.y[data.train_mask]).numpy()
yy = (data.y[data.test_mask]).numpy()
model = GCN2()
optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4)
model.train()
for epoch in range(400):
    optimizer.zero_grad()
    out = model(data)
    loss = F.nll_loss(out[data.train_mask], data.y[data.train_mask])
    loss.backward()
    optimizer.step()
model.eval()
GCN2(
  (conv1): GCNConv(1, 32)
  (conv2): GCNConv(32, 2)
)
pred = model(data).argmax(dim=1)
yyhat = pred[data.test_mask]

metrics = [sklearn.metrics.accuracy_score,
           sklearn.metrics.precision_score,
           sklearn.metrics.recall_score,
           sklearn.metrics.f1_score]

_results1= pd.DataFrame({m.__name__:[m(yy,yyhat).round(6)] for m in metrics},index=['분석1'])
_results1
/home/coco/anaconda3/envs/py38/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1344: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
accuracy_score precision_score recall_score f1_score
분석1 0.494838 0.0 0.0 0.0 
b,W = list(model.conv1.parameters())
b,W
(Parameter containing:
 tensor([-2.6334e-02, -2.7689e-01, -1.6926e-02, -5.2382e-02, -1.2667e-01,
         -4.2815e-02, -2.6620e-02,  1.1360e-01, -2.8240e-02, -2.1838e-01,
          9.7086e-02, -1.2640e-02, -1.5231e-02, -1.3934e-03, -7.6017e-03,
         -3.6946e-02, -5.4102e-10, -1.9268e-01,  1.7090e-07, -8.3471e-10,
         -2.0092e-01, -7.3827e-02, -4.7944e-02, -1.9878e-01, -2.0639e-01,
         -2.7078e-01, -5.2340e-03, -3.9832e-02,  3.9550e-02,  5.1139e-01,
         -3.2245e-02,  2.2552e-02], requires_grad=True),
 Parameter containing:
 tensor([[ 0.1231],
         [ 0.1348],
         [-0.0504],
         [ 0.0830],
         [-0.0851],
         [-0.0475],
         [-0.0506],
         [-0.0098],
         [ 0.2435],
         [ 0.1998],
         [-0.0074],
         [-0.0560],
         [ 0.1415],
         [-0.0649],
         [-0.0586],
         [-0.0573],
         [-0.0653],
         [ 0.0411],
         [-0.0703],
         [-0.0537],
         [ 0.1198],
         [-0.0690],
         [-0.0654],
         [ 0.2001],
         [-0.0048],
         [-0.0085],
         [-0.0613],
         [-0.0797],
         [ 0.1049],
         [-0.0017],
         [-0.0547],
         [ 0.1147]], requires_grad=True))
import networkx as nx
import matplotlib.pyplot as plt

# PyTorch Geometric 그래프를 NetworkX 그래프로 변환
G = torch_geometric.utils.to_networkx(data, to_undirected=True)

# 노드 1023과 연결된 모든 노드와 엣지를 포함하는 부분 그래프 추출
subgraph1 = nx.ego_graph(G, 1023, radius=1)
subgraph2 = nx.ego_graph(G, 9782, radius=1)

# 왼쪽 그래프 레이아웃 설정
pos1 = nx.spring_layout(subgraph1, pos=None, seed=42)  # 그래프 레이아웃 설정

# 오른쪽 그래프 레이아웃 설정
pos2 = nx.spring_layout(subgraph2, pos=None, seed=43)

# 노드 그리기
nx.draw_networkx_nodes(subgraph1, pos1, node_size=200, node_color='b', label='Node 1023')
nx.draw_networkx_nodes(subgraph2, pos2, node_size=200, node_color='g', label='Node 9782')

# 엣지 그리기
nx.draw_networkx_edges(subgraph1, pos1)
nx.draw_networkx_edges(subgraph2, pos2)

# 노드 인덱스 표시
labels1 = {node: str(node) for node in subgraph1.nodes()}
labels2 = {node: str(node) for node in subgraph2.nodes()}
nx.draw_networkx_labels(subgraph1, pos1, labels1, font_size=10)
nx.draw_networkx_labels(subgraph2, pos2, labels2, font_size=10)

# 그래프 출력
plt.axis('off')  # 축 숨기기

# 레이블 표시
plt.legend(loc='best')

plt.show()

![](231020 데이터(19, df5_files/figure-html/cell-28-output-1.png){}

# 노드 1023와 9782 간의 연결 확인
are_connected = G.has_edge(1023, 9782)

if are_connected:
    print("Node 1023 and Node 9782 are connected.")
else:
    print("Node 1023 and Node 9782 are not connected.")
Node 1023 and Node 9782 are not connected.
G.has_edge(1023,1024)
True